\documentclass[10pt,oneside]{book}

\input{macros.tex}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{document}

\pagestyle{empty}
\thispagestyle{empty}


For action ``wash'':

\begin{center}
\begin{tikzpicture}[->]
\node[state] (dirty) {{\em dirty}};
\node[state, right of=dirty, xshift=2.5cm] (clean) {{\em clean}};
\node[state, below of=dirty, yshift=-2.5cm] (painted) {{\em painted}};
\node[state, below of=clean, yshift=-2.5cm] (ejected) {{\em ejected}};
\draw (dirty) edge[loop above] node{0.1} (dirty)
      (dirty) edge[bend left, above] node{0.9} (clean)
      (clean) edge[loop above] node{0.9} (clean)
      (clean) edge[bend left, below] node{0.1} (dirty)
      (painted) edge[left] node{0.1} (dirty)
      (painted) edge[right] node{~0.9} (clean)
      (ejected) edge[loop above] node{1.0} (ejected);
\end{tikzpicture}
\end{center}

\bigskip

For action ``paint'':

\begin{center}
\begin{tikzpicture}[->]
\node[state] (dirty) {{\em dirty}};
\node[state, right of=dirty, xshift=2.5cm] (clean) {{\em clean}};
\node[state, below of=dirty, yshift=-2.5cm] (painted) {{\em painted}};
\node[state, below of=clean, yshift=-2.5cm] (ejected) {{\em ejected}};
\draw (dirty) edge[loop above] node{1} (dirty)
      (clean) edge[loop above] node{0.1} (clean)
      (clean) edge[left, below] node{0.1} (dirty)
      (clean) edge[left, below] node{0.8} (painted)
      (painted) edge[loop above] node{1} (painted)
      (ejected) edge[loop above] node{1} (ejected);
\end{tikzpicture}
\end{center}

\bigskip

For action ``eject'':

\begin{center}
\begin{tikzpicture}[->]
\node[state] (dirty) {{\em dirty}};
\node[state, right of=dirty, xshift=2.5cm] (clean) {{\em clean}};
\node[state, below of=dirty, yshift=-2.5cm] (painted) {{\em painted}};
\node[state, below of=clean, yshift=-2.5cm] (ejected) {{\em ejected}};
\draw (dirty) edge[right, above] node{1} (ejected)
      (clean) edge[right] node{1} (ejected)
      (painted) edge[right, above] node{1} (ejected)
      (ejected) edge[loop right] node{1} (ejected);
\end{tikzpicture}
\end{center}

\newpage
For deterministic machine:

\begin{center}
\begin{tikzpicture}[->]
\node[state] (dirty) {{\em dirty}};
\node[state, right of=dirty, xshift=2.5cm] (clean) {{\em clean}};
\node[state, below of=dirty, yshift=-2.5cm] (painted) {{\em painted}};
\node[state, below of=clean, yshift=-2.5cm] (ejected) {{\em ejected}};
\draw (dirty) edge[left, above] node{wash} (clean)
      (dirty) edge[bend left, above] node{eject} (ejected)
      (dirty) edge[loop left] node{paint} (dirty)
      (clean) edge[bend right, below] node{paint} (painted)
      (clean) edge[right, below] node{eject} (ejected)
      (clean) edge[loop right] node{wash} (clean)
      (painted) edge[bend right, below] node{wash} (clean)
      (painted) edge[loop left] node{paint} (painted)
      (painted) edge[right, below] node{eject} (ejected)
      (ejected) edge[loop right] node{any action} (ejected);
\end{tikzpicture}
\end{center}


\end{document}
